<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2015-2017 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Ingestion and Exploration</title>

    <link rel="stylesheet" href="../../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../../_static/jquery.js"></script>
    <script type="text/javascript" src="../../_static/underscore.js"></script>
    <script type="text/javascript" src="../../_static/doctools.js"></script>
    <script type="text/javascript" src="../../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../../genindex.html" />
    <link rel="search" title="Search" href="../../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../../index.html" />
    <link rel="up" title="Cloudera" href="index.html" />
    <link rel="next" title="&lt;no title&gt;" href="configuring.html" />
    <link rel="prev" title="Cloudera" href="index.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: integrations -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><a href="../../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a>
      </li>
      <li class="toctree-l1"><a href="../../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../index.html"> Introduction</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../hub.html"> Hub</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> Cloudera</a><ul class="current">
<li class="toctree-l2 current"><a class="current reference internal" href="#"> Ingestion and Exploration</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../apache-sentry.html"> Apache Sentry</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../apache-ranger.html"> Apache Ranger</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../hadoop-kms.html"> Apache Hadoop KMS</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../jdbc.html"> CDAP JDBC Driver</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../odbc.html"> CDAP ODBC Driver</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../pentaho.html"> Pentaho Data Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../squirrel.html"> SquirrelSQL</a></li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="ingestion-and-exploration">
<span id="cloudera-ingesting"></span><h1>Ingestion and Exploration<a class="headerlink" href="#ingestion-and-exploration" title="Permalink to this headline">🔗</a></h1>
<div class="section" id="integrating-cdap-with-impala">
<span id="integrations-impala"></span><h2>Integrating CDAP with Impala<a class="headerlink" href="#integrating-cdap-with-impala" title="Permalink to this headline">🔗</a></h2>
<p>When using CDAP with Impala:</p>
<ul class="simple">
<li><a class="reference external" href="../../../developer-manual/pipelines/index.html"><strong>数据流管道:</strong></a> Using <strong>CDAP pipelines</strong> and <strong>plugins</strong> to create either real-time or batch data pipelines</li>
</ul>
<ul class="simple">
<li><a class="reference external" href="../../../developer-manual/data-exploration/streams.html"><strong>Stream Exploration:</strong></a> It is often useful to be able to <strong>examine data in a stream</strong> in an ad-hoc manner through SQL-like queries</li>
</ul>
<ul class="simple">
<li><a class="reference external" href="../../../developer-manual/data-exploration/filesets.html"><strong>Fileset Exploration:</strong></a> The <strong>CDAP FileSet Datasets</strong> can be explored through ad-hoc SQL-like queries.</li>
</ul>
</div>
<div class="section" id="ingesting-and-exploring-data-with-impala">
<h2>Ingesting and Exploring Data with Impala<a class="headerlink" href="#ingesting-and-exploring-data-with-impala" title="Permalink to this headline">🔗</a></h2>
<p>Streams are the primary means of bringing data from external systems into the CDAP in
real time. They are ordered, time-partitioned sequences of data, usable for real-time
collection and consumption of data.</p>
<p>They can easily be created by using the CDAP Command Line Interface (CLI).
First, connect to your CDAP instance using the CLI:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">cdap &gt; connect &lt;hostname&gt;:11015</span>
</pre></div>
</div>
<p>Next, create a stream:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">cdap &gt; create stream trades</span>
</pre></div>
</div>
<p>You can then add events to a stream, one-by-one:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">cdap &gt; send stream trades &#39;NFLX,441.07,50&#39;</span>
<span class="go">cdap &gt; send stream trades &#39;AAPL,118.63,100&#39;</span>
<span class="go">cdap &gt; send stream trades &#39;GOOG,528.48,10&#39;</span>
</pre></div>
</div>
<p>Or, you can add the entire contents of a file:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">cdap &gt; load stream trades &lt;my_path&gt;/trades.csv</span>
</pre></div>
</div>
<p>Or, you can use the other tools and APIs available to ingest data in real time or batch.
For more information on what are other ways of ingesting data into CDAP, see the links at
the top of this page.</p>
<p>You can now examine the contents of your stream by executing a SQL query:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">cdap &gt; execute &#39;select * from stream_trades limit 5&#39;</span>
<span class="go">+===================================================================================================+</span>
<span class="go">| stream_trades.ts: BIGINT | stream_trades.headers: map&lt;string,string&gt; | stream_trades.body: STRING |</span>
<span class="go">+===================================================================================================+</span>
<span class="go">| 1422493022983            | {}                                        | NFLX,441.07,50             |</span>
<span class="go">| 1422493027358            | {}                                        | AAPL,118.63,100            |</span>
<span class="go">| 1422493031802            | {}                                        | GOOG,528.48,10             |</span>
<span class="go">| 1422493036080            | {&quot;content.type&quot;:&quot;text/csv&quot;}               | GOOG,538.53,18230          |</span>
<span class="go">| 1422493036080            | {&quot;content.type&quot;:&quot;text/csv&quot;}               | GOOG,538.17,100            |</span>
<span class="go">+===================================================================================================+</span>
</pre></div>
</div>
<p>You can also attach a schema to your stream to enable more powerful queries:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">cdap &gt; set stream format trades csv &#39;ticker string, price double, trades int&#39;</span>
<span class="go">cdap &gt; execute &#39;select ticker, sum(price * trades) / 1000000 as millions from stream_trades group by ticker order by millions desc&#39;</span>
<span class="go">+=====================================+</span>
<span class="go">| ticker: STRING | millions: DOUBLE   |</span>
<span class="go">+=====================================+</span>
<span class="go">| AAPL           | 3121.8966341143905 |</span>
<span class="go">| NFLX           | 866.0789117408007  |</span>
<span class="go">| GOOG           | 469.01340359839986 |</span>
<span class="go">+=====================================+</span>
</pre></div>
</div>
<p>On one of our test clusters, the above query took just about two minutes to complete.
Data in CDAP is integrated with Apache Hive, and the above query above translates to a Hive query.
As such, it will launch two MapReduce jobs in order to calculate the query results, which
is why it takes minutes instead of seconds.</p>
<p>To reduce query time, you can use Impala to query the data instead of Hive. Since streams
are written in a custom format, they cannot be directly queried through Impala. Instead,
you can create an ETL batch application that regularly reads
stream events and writes those events into files on HDFS that can then be queried by Impala.</p>
<p>To do this, write the following JSON to a config file:</p>
<pre class="copyable copyable-text literal-block">
{
  &quot;description&quot;: &quot;Periodically reads stream data and writes it to a TimePartitionedFileSet&quot;,
  &quot;artifact&quot;: {
    &quot;name&quot;: &quot;cdap-data-pipeline&quot;,
    &quot;scope&quot;: &quot;system&quot;,
    &quot;version&quot;: &quot;6.1.1&quot;
  },
  &quot;config&quot;: {
    &quot;schedule&quot;: &quot;*/10 * * * *&quot;,
    &quot;engine&quot;: &quot;mapreduce&quot;,
    &quot;stages&quot;: [
      {
        &quot;name&quot;: &quot;tradeStream&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Stream&quot;,
          &quot;type&quot;: &quot;batchsource&quot;,
          &quot;properties&quot;: {
            &quot;name&quot;: &quot;trades&quot;,
            &quot;duration&quot;: &quot;10m&quot;,
            &quot;format&quot;: &quot;csv&quot;,
            &quot;schema&quot;: &quot;{
              \&quot;type\&quot;:\&quot;record\&quot;,
              \&quot;name\&quot;:\&quot;purchase\&quot;,
              \&quot;fields\&quot;:[
                {\&quot;name\&quot;:\&quot;ticker\&quot;,\&quot;type\&quot;:\&quot;string\&quot;},
                {\&quot;name\&quot;:\&quot;price\&quot;,\&quot;type\&quot;:\&quot;double\&quot;},
                {\&quot;name\&quot;:\&quot;trades\&quot;,\&quot;type\&quot;:\&quot;int\&quot;}
              ]
            }&quot;,
            &quot;format.setting.delimiter&quot;:&quot;,&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;dropHeadersTransform&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Projection&quot;,
          &quot;type&quot;: &quot;transform&quot;,
          &quot;properties&quot;: {
            &quot;drop&quot;: &quot;headers&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;tpfsAvroSink&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;TPFSAvro&quot;,
          &quot;type&quot;: &quot;batchsink&quot;,
          &quot;properties&quot;: {
            &quot;name&quot;: &quot;trades_converted&quot;,
            &quot;schema&quot;: &quot;{
              \&quot;type\&quot;:\&quot;record\&quot;,
              \&quot;name\&quot;:\&quot;purchase\&quot;,
              \&quot;fields\&quot;:[
                {\&quot;name\&quot;:\&quot;ts\&quot;,\&quot;type\&quot;:\&quot;long\&quot;},
                {\&quot;name\&quot;:\&quot;ticker\&quot;,\&quot;type\&quot;:\&quot;string\&quot;},
                {\&quot;name\&quot;:\&quot;price\&quot;,\&quot;type\&quot;:\&quot;double\&quot;},
                {\&quot;name\&quot;:\&quot;trades\&quot;,\&quot;type\&quot;:\&quot;int\&quot;}
              ]
            }&quot;,
            &quot;basePath&quot;: &quot;trades_converted&quot;
          }
        }
      }
    ],
    &quot;connections&quot;: [
      {
        &quot;from&quot;: &quot;tradeStream&quot;,
        &quot;to&quot;: &quot;dropHeadersTransform&quot;
      },
      {
        &quot;from&quot;: &quot;dropHeadersTransform&quot;,
        &quot;to&quot;: &quot;tpfsAvroSink&quot;
      }
    ]
  }
}
</pre>
<p><strong>Note:</strong> The above JSON has been re-formatted to fit and requires editing (remove the line endings added to
the <code class="docutils literal notranslate"><span class="pre">schema</span></code> values) to be a conforming JSON file.</p>
<p>Then use your config file with the <code class="docutils literal notranslate"><span class="pre">cdap-data-pipeline</span></code> artifact to create an application through the CLI.
For example, if you wrote the above JSON to a file named <code class="docutils literal notranslate"><span class="pre">conversion.json</span></code>:</p>
<div class="highlight container">
<pre class="literal-block">
cdap &gt; create app trades_conversion cdap-data-pipeline 6.1.1 system &lt;path-to-conversion.json&gt;
</pre>
</div>
<p>This will create and configure an application. The application’s schedule (named, by default, to <code class="docutils literal notranslate"><span class="pre">dataPipelineSchedule</span></code>)
will not run until you resume it:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">cdap &gt; resume schedule trades_conversion.dataPipelineSchedule</span>
</pre></div>
</div>
<p>This will start a schedule that will run the workflow every ten minutes.
The next time the workflow runs, it will spawn a MapReduce job that reads all events added
in the past ten minutes, writes each event to Avro encoded files, and registers a new
partition in the Hive Metastore. We can then query the contents using Impala. On a
cluster, use the Impala shell to connect to Impala:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> impala-shell -i &lt;impala-host&gt;
<span class="gp">&gt;</span> invalidate metadata
<span class="gp">&gt;</span> <span class="k">select</span> ticker, sum<span class="o">(</span>price * trades<span class="o">)</span> / <span class="m">1000000</span> as millions from dataset_trades_converted group by ticker order by millions desc
<span class="go">+--------+-------------------+</span>
<span class="go">| ticker | millions          |</span>
<span class="go">+--------+-------------------+</span>
<span class="go">| AAPL   | 3121.88477111439  |</span>
<span class="go">| NFLX   | 866.0568582408006 |</span>
<span class="go">| GOOG   | 469.0081187983999 |</span>
<span class="go">+--------+-------------------+</span>
<span class="go">Fetched 3 row(s) in 1.03s</span>
</pre></div>
</div>
<p>Since we are using Impala, no MapReduce jobs are launched, and the query comes back in
about one second.</p>
<p>Now that you have data in CDAP and are able to explore your data, you can use CDAP’s many
useful and powerful services, such as the ability to dynamically scale processing units,
distributed transactions, and service discovery, to write applications that meet your
business needs.</p>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Cloudera" href="index.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right">&mdash;</div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../../_static/js/js.cookie.js"></script><script type="text/javascript" src="../../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>